from pymongo import MongoClient
from bs4 import BeautifulSoup
from pprint import pprint
import requests
import datetime
import json
import time
import re

server_client = MongoClient('localhost', 27017)

server_db = server_client['knx_posts_db']
offical_posts_coll = server_db['offical_posts_coll']


class POST():
    def __init__(self):
        self.params = {

        }
        self.data = {

        }
        self.headers = {

        }
        self.scrapy()

    def scrapy(self):
        r = requests.get('http://www.nyckidsclub.com/about/join/')
        r.encoding = 'utf-8'
        soup = BeautifulSoup(r.text)

        for jname in soup.select('.zpzwBOX a'):
            url = jname.get('href')
            r = requests.get(url)
            r.encoding = 'utf-8'

            soup = BeautifulSoup(r.text)

            name = soup.find(class_ = 'zwBOX1_p2').text
            description = soup.find(class_ = 'zwms_BOX').get_text()
            location = '北京'

            item = {
                "url": url,
                'edu': '',
                'exp': [],
                'name': name,
                'date': '',
                'lang': '',
                'place': '',
                'major': '',
                'count': -1,
                'salary': [],
                'toSchool': True,
                'welfare': [],
                'funType': '',
                'company': '纽约国际儿童俱乐部',
                'location': location,
                'industry': '教育',
                'keywords': [],
                'platform': 'offical',
                'searchKeyword': '',
                'description': description,
                'subIndustry': '',
                'stime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            }

            result = offical_posts_coll.replace_one({'company': '纽约国际儿童俱乐部', 'name': name, 'location': location}, item, True)

            if result.matched_count:
                print('-' * 40, 'update one job', '-' * 40)
            else:
                print('-' * 40, 'insert one job', '-' * 40)

            pprint(item)


POST()
